In [1]:
# Few base imports and logging.

import logging
import functools
import itertools

import numpy as np
logging.basicConfig()
logger = logging.getLogger('vd')
logger.setLevel(logging.DEBUG)

# make sure you have the pep8_magic installed
# jupyter nbextension install --user pep8_magic.py
%load_ext pep8_magic
In [2]:
import matplotlib.pyplot as plt
%matplotlib inline
import os
import numpy as np
from skimage.exposure import equalize_adapthist
from skimage.transform import resize
from skimage.color import rgb2ycbcr
from scipy.misc import imsave
import cv2
import glob

# Few helper functions
def show_images(images,titles=None, save=None):
    """Display a list of images"""
    n_ims = len(images)
    if titles is None: titles = ['(%d)' % i for i in range(1,n_ims + 1)]
    fig = plt.figure()
    n = 1
    for image,title in zip(images,titles):
        a = fig.add_subplot(1,n_ims,n) # Make subplot
        if len(image.shape) == 2 or image.shape[2] == 1: # Is image grayscale?
            plt.imshow(np.resize(image, (image.shape[0], image.shape[1])), interpolation="bicubic", cmap="gray") # Only place in this blog you can't replace 'gray' with 'grey'
        else:
            plt.imshow(image, interpolation="bicubic")
        if titles is not None:
            a.set_title(title)
        n += 1
    fig.set_size_inches(np.array(fig.get_size_inches()) * n_ims)
    plt.tight_layout()
    plt.show()
    if save is not None:
        fig.savefig("output_images/" + save + ".png")
        

def extract_frames(clip, times, imgdir, imgname):
    for t in times:
        imgpath = os.path.join(imgdir, '{}-{}.jpg'.format(imgname, t))
        clip.save_frame(imgpath, t)
        
def resize(image, c=(32, 32)):
    return cv2.resize(image, c)

def img_color_spaces(img_rgb):
    return [(img_rgb, "RGB"), (cv2.cvtColor(img_rgb, cv2.COLOR_RGB2HSV), "HSV"),
            (cv2.cvtColor(img_rgb, cv2.COLOR_RGB2LUV), "LUV"),
            (cv2.cvtColor(img_rgb, cv2.COLOR_RGB2HLS), "HLS"),
            (cv2.cvtColor(img_rgb, cv2.COLOR_RGB2YUV), "YUV"),
            (cv2.cvtColor(img_rgb, cv2.COLOR_RGB2YCrCb), "YCrCb")]

def load_img(img_fname):
    return cv2.cvtColor(cv2.imread(img_fname,  cv2.IMREAD_COLOR), cv2.COLOR_BGR2RGB)
                
def draw_box(img_fname, c1, c2):
    return cv2.rectangle(load_img(img_fname), c1, c2, (0,255,0), 4)
    
def output_image(img, img_name, prefix="output_images"):
    imsave(str(prefix) + "/" + str(img_name) + ".png", img)
    
def get_rnd(l):
    return l[np.random.randint(len(l), size=1)[0]]

Download the provided data

This code block downloads the data and unzip into given location. This is optionally run.

The data sets used are vehicle and non-vehicle.

Optionally the larger data 1 set provided here from self-driving-car/annotations/

In [3]:
#%%pep8

import urllib.request
import urllib.parse
import urllib.error

from pathlib import Path


def download_data(from_url, file_name, to_dir="data", retry=False):
    to_path = '{}/{}'.format(str(to_dir), str(file_name))
    if retry is True or Path(to_path).is_file() is False:
        logger.info("downloading {}".format(from_url))
        f = urllib.request.urlopen(from_url)
        data = f.read()
        with open(to_path, "wb") as d:
            logger.info("writing data to: {}".format(to_path))
            d.write(data)
download_data(
    "https://s3.amazonaws.com/udacity-sdc/Vehicle_Tracking/vehicles.zip",
    "vehicles.zip")
download_data(
    "https://s3.amazonaws.com/udacity-sdc/Vehicle_Tracking/non-vehicles.zip",
    "non-vehicles.zip")
download_data(
    "https://github.com/udacity/self-driving-car/blob/master/annotations/labels_crowdai.csv",
    "labels_crowdai.csv")
download_data(
    "http://bit.ly/udacity-annoations-crowdai",
    "object-detection-crowdai.tar")

Extract the data

This code block extracts the above downloaded data.

In [4]:
#%%pep8

import zipfile
import tarfile

data_files = ["vehicles.zip", "non-vehicles.zip", "object-detection-crowdai.tar"]

def extract_file(zfile, prefix_dir="data", retry=False):
    tdir = zfile.split('.')[0]
    f = '{}/{}'.format(str(prefix_dir), str(zfile))
    to_path = '{}'.format(str(prefix_dir))
    check_path = '{}/{}'.format(to_path, tdir)
    if retry is True or Path(check_path).is_dir() is False:
        logger.info("Extracting {} to {}/".format(f, to_path))
        zip_ref = None
        if (f.endswith("tar.gz")):
            zip_ref = tarfile.open(f, "r:gz")
        elif (f.endswith("tar")):
            zip_ref = tarfile.open(f, "r:")
        elif (f.endswith("zip")):
            zip_ref = zipfile.ZipFile(f, "r")
    
        if zip_ref is None:
            err_str = "Invalid compressed file: {}".format(f)
            logger.error(err_str)
            raise Exception(err_str)

        zip_ref.extractall(to_path)
        zip_ref.close()
    return check_path

data_dirs = [ extract_file(x) for x in data_files ]

Load and inspect the default dataset provided

Show few examples and see how they look like

In [5]:
vehicle_images = [load_img(x) for x in glob.glob("{}/*/*.png".format(data_dirs[0]))]
non_vehicle_images = [load_img(x) for x in glob.glob("{}/*/*.png".format(data_dirs[1]))]
print("Number if images in dataset[{}] is {}".format(data_dirs[0], len(vehicle_images)))
print("Number if images in dataset[{}] is {}".format(data_dirs[1], len(non_vehicle_images)))
print("Shape of image: {}".format(vehicle_images[0].shape))
Number if images in dataset[data/vehicles] is 8792
Number if images in dataset[data/non-vehicles] is 8968
Shape of image: (64, 64, 3)
In [6]:
from skimage.feature import hog
from skimage import data, color, exposure

def get_hog_features(img, orient, pix_per_cell, cell_per_block,
                     vis=False, feature_vec=True):
    # Call with two outputs if vis==True
    if vis == True:
        features, hog_image = hog(img, orientations=orient, 
                                  pixels_per_cell=(pix_per_cell, pix_per_cell),
                                  cells_per_block=(cell_per_block, cell_per_block), 
                                  transform_sqrt=False, 
                                  visualise=vis, feature_vector=feature_vec)
        return features, hog_image
    # Otherwise call with one output
    else:      
        features = hog(img, orientations=orient, 
                       pixels_per_cell=(pix_per_cell, pix_per_cell),
                       cells_per_block=(cell_per_block, cell_per_block), 
                       transform_sqrt=False, 
                       visualise=vis, feature_vector=feature_vec)
        return features

def get_hog(img, orient=9, pix_per_cell=8, cell_per_block=2, visualise=False,
            feature_vector=True, hog_channel="ALL"):
    if visualise is False:
        hog_features = []
        if hog_channel == "ALL":
            # Compute individual channel HOG features for the entire image
            hog1 = get_hog_features(img[:,:,0], orient, pix_per_cell, cell_per_block)
            hog2 = get_hog_features(img[:,:,1], orient, pix_per_cell, cell_per_block)
            hog3 = get_hog_features(img[:,:,2], orient, pix_per_cell, cell_per_block)
            hog_features = np.hstack((hog1, hog2, hog3))
        else:
            hog_features = get_hog_features(img[:,:,hog_channel], orient, pix_per_cell, cell_per_block)
        return hog_features
    else:
        slist = cv2.split(img)
        #slist = [cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)]
        return [hog(x, orientations=orient, pixels_per_cell=(pix_per_cell, pix_per_cell),
                   cells_per_block=(cell_per_block, cell_per_block), transform_sqrt=False, visualise=True, 
                    feature_vector=False)[1] for x in slist]
    

def rescale_intensity(hog_image):
    # Rescale histogram for better display
    return exposure.rescale_intensity(hog_image, in_range=(0, 0.02))
In [7]:
image = vehicle_images[1]
images = img_color_spaces(image)

for x in images:
    l = [image]
    t = ["orig"]
    t.extend([x for x in x[1]])
    hogs = [k for k in get_hog(x[0], visualise=True)]
    l.extend(hogs)
    show_images(l, t, save='hog_{}'.format(x[1]))
    
def plot_hog(img, title):
    f = plt.figure()
    plt.subplot(211)
    plt.imshow(img, interpolation="bicubic")
    plt.subplot(212)
    plt.plot(get_hog((img)))
    f.suptitle(title, fontsize=10)
    f.tight_layout()
    plt.show()
    f.savefig("output_images/hog_YCrCb_features.png")

plot_hog(image, "vehicle")
/home/powell/ENTER/envs/carnd-term1/lib/python3.5/site-packages/skimage/feature/_hog.py:119: skimage_deprecation: Default value of `block_norm`==`L1` is deprecated and will be changed to `L2-Hys` in v0.15
  'be changed to `L2-Hys` in v0.15', skimage_deprecation)

Process additional data

Crop and resize additional data

The additional large data 1 set provided here from self-driving-car/annotations/ needs cropping based on the data in the labels.csv file.

Example:

xmin,xmax,ymin,ymax,Frame,Label,Preview URL
785,533,905,644,1479498371963069978.jpg,Car,http://crowdai.com/images/Wwj-gorOCisE7uxA/visualize
89,551,291,680,1479498371963069978.jpg,Car,http://crowdai.com/images/Wwj-gorOCisE7uxA/visualize
In [8]:
#%%pep8

import csv


def process_csv(prefix, fname,
                row_labels=('Frame', 'Label', 'xmin',
                            'xmax', 'ymin', 'ymax')):
    fpath = '{}/{}'.format(prefix, "labels.csv")
    with open(fpath) as csvfile:
        reader = csv.DictReader(csvfile)
        ret = []
        for row in reader:
            v = ['{}/{}'.format(prefix, row['Frame']), row['Label']]
            v.append((int(row['xmin']), int(row['xmax'])))  # xmax is actually ymin
            v.append((int(row['ymin']), int(row['ymax'])))  # ymin is actually xmax
            ret.append(v)
    return ret, row_labels


r, header = process_csv(data_dirs[2], 'labels.csv')

logger.debug(header)
logger.debug(r[0])
logger.debug(r[1])
DEBUG:vd:('Frame', 'Label', 'xmin', 'xmax', 'ymin', 'ymax')
DEBUG:vd:['data/object-detection-crowdai/1479498371963069978.jpg', 'Car', (785, 533), (905, 644)]
DEBUG:vd:['data/object-detection-crowdai/1479498371963069978.jpg', 'Car', (89, 551), (291, 680)]
In [9]:
# Show size of the data set
print("Number of samples: {}".format(len(r)))

# Shape of a image
print("Shape of image: {}".format(load_img(r[0][0]).shape))

# Unique classes
print("Classes: {}".format(np.unique([x[1] for x in r])))

# Draw boxes on the first two images.
img_idxs = [ 0, 1 ]

imgs = [ draw_box(r[i][0], r[i][2], r[i][3]) for i in img_idxs ]
imgs_label = [ r[i][1] for i in img_idxs ]

show_images(imgs, imgs_label)

# Pick few random images and draw a rectange around the object
img_idxs = np.random.choice(len(r), 4)
imgs = [ draw_box(r[i][0], r[i][2], r[i][3]) for i in img_idxs ]
imgs_label = [ r[i][1] for i in img_idxs ]

show_images(imgs, imgs_label)
Number of samples: 72064
Shape of image: (1200, 1920, 3)
Classes: ['Car' 'Pedestrian' 'Truck']

Visualization of given data

Explore various color spaces

In [10]:
from mpl_toolkits.mplot3d import Axes3D

def plot3d(pixels, colors_rgb,
        axis_labels=list("RGB"), axis_limits=((0, 255), (0, 255), (0, 255))):
    """Plot pixels in 3D."""

    # Create figure and 3D axes
    fig = plt.figure(figsize=(8, 8))
    ax = Axes3D(fig)

    # Set axis limits
    ax.set_xlim(*axis_limits[0])
    ax.set_ylim(*axis_limits[1])
    ax.set_zlim(*axis_limits[2])

    # Set axis labels and sizes
    ax.tick_params(axis='both', which='major', labelsize=14, pad=8)
    ax.set_xlabel(axis_labels[0], fontsize=16, labelpad=16)
    ax.set_ylabel(axis_labels[1], fontsize=16, labelpad=16)
    ax.set_zlabel(axis_labels[2], fontsize=16, labelpad=16)

    # Plot pixel values with colors given in colors_rgb
    ax.scatter(
        pixels[:, :, 0].ravel(),
        pixels[:, :, 1].ravel(),
        pixels[:, :, 2].ravel(),
        c=colors_rgb.reshape((-1, 3)), edgecolors='none')

    return ax  # return Axes3D object for further manipulation
In [11]:
def plot_color_spaces(img, img_title):
    # Select a small fraction of pixels to plot by subsampling it
    scale = max(img.shape[0], img.shape[1], 64) / 64  # at most 64 rows and columns
    img_small = cv2.resize(img, (np.int(img.shape[1] / scale), np.int(img.shape[0] / scale)), interpolation=cv2.INTER_NEAREST)

    # Convert subsampled image to desired color space(s)
    img_small_RGB = img_small
    img_small_HSV = cv2.cvtColor(img_small, cv2.COLOR_RGB2HSV)
    img_small_LUV = cv2.cvtColor(img_small, cv2.COLOR_RGB2LUV)
    img_small_YCrCb = cv2.cvtColor(img_small, cv2.COLOR_RGB2YCrCb)
    img_small_rgb = img_small_RGB / 255.  # scaled to [0, 1], only for plotting

    show_images([img], [img_title])
    # Plot and show
    plot3d(img_small_RGB, img_small_rgb)
    plt.title("RGB")
    plt.show()

    plot3d(img_small_HSV, img_small_rgb, axis_labels=list("HSV"))
    plt.title("HSV")
    plt.show()
    
    plot3d(img_small_LUV, img_small_rgb, axis_labels=list("LUV"))
    plt.title("LUV")
    plt.show()
    
    plot3d(img_small_YCrCb, img_small_rgb, axis_labels=list("Yrb"))
    plt.title("YCrCb")
    plt.show()
In [12]:
plot_color_spaces(get_rnd(vehicle_images), "vehicle")
In [13]:
plot_color_spaces(get_rnd(non_vehicle_images), "non-vehicle")

Color histogram for various color spaces

In [14]:
def color_hist(img, nbins=32, bins_range=(0, 256)):
    rhist = np.histogram(img[:,:,0], bins=nbins, range=bins_range)
    ghist = np.histogram(img[:,:,1], bins=nbins, range=bins_range)
    bhist = np.histogram(img[:,:,2], bins=nbins, range=bins_range)
    # Generating bin centers
    bin_edges = rhist[1]
    bin_centers = (bin_edges[1:]  + bin_edges[0:len(bin_edges)-1])/2
    # Concatenate the histograms into a single feature vector
    hist_features = np.concatenate((rhist[0], ghist[0], bhist[0]))
    # Return the individual histograms, bin_centers and feature vector
    return rhist, ghist, bhist, bin_centers, hist_features

def plot_color_hist(image, title, color_space="RGB"):
    rh, gh, bh, bincen, feature_vec = color_hist(image, nbins=32, bins_range=(0, 256))
    i, j, k = cv2.split(image)
    if rh is not None:
        fig = plt.figure(figsize=(12,8))
        plt.subplot(231)
        plt.imshow(i, cmap='gray', interpolation="bicubic")
        plt.title('{}'.format(color_space[0]))
        plt.subplot(232)
        plt.imshow(j, cmap='gray', interpolation="bicubic")
        plt.title('{}'.format(color_space[1]))
        plt.subplot(233)
        plt.imshow(k, cmap='gray', interpolation="bicubic")
        plt.title('{}'.format(color_space[2]))
        plt.subplot(234)
        plt.bar(bincen, rh[0])
        plt.xlim(0, 256)
        plt.title('{} Histogram'.format(color_space[0]))
        plt.subplot(235)
        plt.bar(bincen, gh[0])
        plt.xlim(0, 256)
        plt.title('{} Histogram'.format(color_space[1]))
        plt.subplot(236)
        plt.bar(bincen, bh[0])
        plt.xlim(0, 256)
        plt.title('{} Histogram'.format(color_space[2]))
        fig.suptitle(title, fontsize=10)
        fig.tight_layout()
        fig.savefig("output_images/{vehicle_}{}_hist.png".format(title, color_space))
    else:
        logger.error("None returned by color_hist for image")
        
def plot_color_spaces_hist(img, title):
    [plot_color_hist(x[0], title, x[1]) for x in img_color_spaces(img)]
    
In [15]:
plot_color_spaces_hist(get_rnd(vehicle_images), "vehicle")
In [16]:
plot_color_spaces_hist(get_rnd(non_vehicle_images), "non-vehicle")

Spatial Binning of Color

In [17]:
def bin_spatial(img, convert=False, color_space='YCrCb', size=(32, 32)):
    # Convert image to new color space (if specified)
    if convert is True and color_space != 'RGB':
        if color_space == 'HSV':
            feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
        elif color_space == 'LUV':
            feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2LUV)
        elif color_space == 'HLS':
            feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2HLS)
        elif color_space == 'YUV':
            feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2YUV)
        elif color_space == 'YCrCb':
            feature_image = cv2.cvtColor(img, cv2.COLOR_RGB2YCrCb)
    else: feature_image = np.copy(img)             
    # Use cv2.resize().ravel() to create the feature vector
    features = cv2.resize(feature_image, size).ravel() 
    # Return the feature vector
    return features

def plot_bin_spatial(img, title):
    f = plt.figure()
    plt.subplot(211)
    plt.imshow(img, interpolation="bicubic")
    plt.subplot(212)
    plt.plot(bin_spatial(img, convert=True, color_space='YCrCb'))
    f.suptitle(title, fontsize=10)
    f.tight_layout()
    f.savefig("output_images/{}_{}_spatial.png".format(title, 'YCrCb'))
In [18]:
plot_bin_spatial(get_rnd(vehicle_images), "vehicle")
In [19]:
plot_bin_spatial(get_rnd(non_vehicle_images), "non-vehicle")

Combine features

As mentioned in the lesson combine various feature vectors and use StandardScaler() to scale them.

In [20]:
def extract_features(imgs, cspace='YCrCb', spatial_size=(32, 32),
                        hist_bins=32, hist_range=(0, 256)):
    # Create a list to append feature vectors to
    features = []
    # Iterate through the list of images
    for image in imgs:
        # apply color conversion if other than 'RGB'
        if cspace != 'RGB':
            if cspace == 'HSV':
                feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HSV)
            elif cspace == 'LUV':
                feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2LUV)
            elif cspace == 'HLS':
                feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2HLS)
            elif cspace == 'YUV':
                feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YUV)
            elif cspace == 'YCrCb':
                feature_image = cv2.cvtColor(image, cv2.COLOR_RGB2YCrCb)
        else: feature_image = np.copy(image)      
        # Apply bin_spatial() to get spatial color features
        spatial_features = bin_spatial(feature_image, size=spatial_size)
        # Apply color_hist() also with a color space option now
        _, _, _, _, hist_features = color_hist(feature_image, nbins=hist_bins, bins_range=hist_range)
        # Append the new feature vector to the features list
        hog_features = get_hog(feature_image)
        n = np.concatenate((spatial_features, hist_features, hog_features))
        features.append(n)
    # Return list of feature vectors
    return features
In [21]:
from sklearn.preprocessing import StandardScaler, RobustScaler

def get_features_scaled(cspace='YCrCb', spatial_size=(32, 32), hist_range=(0, 256)):
    vehicle_features = extract_features(vehicle_images, cspace=cspace, spatial_size=spatial_size,
                        hist_bins=32, hist_range=hist_range)
    non_vehicle_features = extract_features(non_vehicle_images, cspace=cspace, spatial_size=spatial_size,
                        hist_bins=32, hist_range=hist_range)
    

    print("vehicle_features shape :{}-{}".format(len(vehicle_features), vehicle_features[0].shape))
    print("non_vehicle_features shape :{}-{}".format(len(non_vehicle_features), non_vehicle_features[0].shape))
    X = np.vstack((vehicle_features, non_vehicle_features)).astype(np.float64) 
    print("X shape :{}".format(X.shape))
    # Fit a per-column scaler
    X_scaler = StandardScaler().fit(X)
    # Apply the scaler to X
    scaled_X = X_scaler.transform(X)
    
    # Define the labels vector
    y = np.hstack((np.ones(len(vehicle_images)), np.zeros(len(non_vehicle_images))))
    return scaled_X, y, X, X_scaler

def plot_scaled(cars, scaled_X, X, index=None):
    if index is None:
        car_ind = np.random.randint(0, len(cars))
    else:
        car_ind = index
    # Plot an example of raw and scaled features
    fig = plt.figure(figsize=(12,4))
    plt.subplot(131)
    plt.imshow(cars[car_ind])
    plt.title('Original Image')
    plt.subplot(132)
    plt.plot(X[car_ind])
    plt.title('Raw Features')
    plt.subplot(133)
    plt.plot(scaled_X[car_ind])
    plt.title('Normalized Features')
    fig.tight_layout()
    fig.savefig("output_images/vehicle_raw_scaled.png")
In [22]:
scaled_X, y, X, scaler_X = get_features_scaled()
/home/powell/ENTER/envs/carnd-term1/lib/python3.5/site-packages/skimage/feature/_hog.py:119: skimage_deprecation: Default value of `block_norm`==`L1` is deprecated and will be changed to `L2-Hys` in v0.15
  'be changed to `L2-Hys` in v0.15', skimage_deprecation)
vehicle_features shape :8792-(8460,)
non_vehicle_features shape :8968-(8460,)
X shape :(17760, 8460)
In [23]:
print(scaler_X.get_params())
{'with_std': True, 'copy': True, 'with_mean': True}
In [24]:
plot_scaled(vehicle_images, scaled_X, X)
In [25]:
plot_scaled(vehicle_images, scaled_X, X, index=0)
In [26]:
plot_scaled(vehicle_images, scaled_X, X, index=2)

Train SVM model

In [27]:
import time
from sklearn.model_selection import train_test_split
from sklearn.svm import SVC, LinearSVC
from sklearn.model_selection import RandomizedSearchCV, GridSearchCV

def split_train_test(scaled_X, y):
    return train_test_split(scaled_X, y, test_size=0.2, random_state=42)

def model_SVM_train(scaled_X, y):
    # Split up data into randomized training and test sets
    X_train, X_test, y_train, y_test = train_test_split(
        scaled_X, y, test_size=0.2, random_state=42)
    #parameters = {'C': [1, 10, 100], 'gamma': [1.0, 0.1, 0.01, 0.001, 0.0001], 'kernel': ['rbf']}
    #parameters = {'C': [10], 'gamma': [0.0001], 'kernel': ['rbf']}
    #svc = SVC()
    # Check the training time for the SVC
    #clf = RandomizedSearchCV(svc, parameters, n_jobs=8)
    #clf = GridSearchCV(svc, parameters)
    #clf = SVC(parameters['C'][0], gamma=parameters['gamma'][0], 
    #          kernel=parameters['kernel'][0])
    clf = LinearSVC()
    t=time.time()
    clf.fit(X_train, y_train)
    t2 = time.time()
    print(round(t2-t, 2), 'Seconds to train SVC...')
    
    return clf, X_test, y_test

def model_SVM_test(clf, X_test, y_test):
    #print("SVM parametrs {}".format(clf.best_params_))
    print('Test Accuracy of SVC = ', round(clf.score(X_test, y_test), 4))
    t=time.time()
    n_predict = 10
    print('My SVC predicts: ', clf.predict(X_test[0:n_predict]))
    print('For these',n_predict, 'labels: ', y_test[0:n_predict])
    t2 = time.time()
    print(round(t2-t, 5), 'Seconds to predict', n_predict,'labels with SVC')
In [28]:
clf, X_test, y_test = model_SVM_train(scaled_X, y)
27.61 Seconds to train SVC...
In [29]:
model_SVM_test(clf, X_test, y_test)
Test Accuracy of SVC =  0.9961
My SVC predicts:  [ 0.  0.  0.  1.  1.  1.  1.  0.  0.  0.]
For these 10 labels:  [ 0.  0.  0.  1.  1.  1.  1.  0.  0.  0.]
0.00114 Seconds to predict 10 labels with SVC
In [30]:
from sklearn.externals import joblib
joblib.dump(clf, 'linear_svm.pkl') 
Out[30]:
['linear_svm.pkl']
In [31]:
from sklearn.externals import joblib
clf = joblib.load('linear_svm.pkl') 
In [32]:
X_train, X_test, y_train, y_test = split_train_test(scaled_X, y)
model_SVM_test(clf, X_test, y_test)
Test Accuracy of SVC =  0.9961
My SVC predicts:  [ 0.  0.  0.  1.  1.  1.  1.  0.  0.  0.]
For these 10 labels:  [ 0.  0.  0.  1.  1.  1.  1.  0.  0.  0.]
0.00132 Seconds to predict 10 labels with SVC

Sliding window implementation

In [33]:
from scipy.ndimage.measurements import label

def add_heat(heatmap, bbox_list):
    # Iterate through list of bboxes
    for box in bbox_list:
        # Add += 1 for all pixels inside each bbox
        # Assuming each "box" takes the form ((x1, y1), (x2, y2))
        heatmap[box[0][1]:box[1][1], box[0][0]:box[1][0]] += 1

    # Return updated heatmap
    return heatmap# Iterate through list of bboxes
    
def apply_threshold(heatmap, threshold):
    # Zero out pixels below the threshold
    heatmap[heatmap <= threshold] = 0
    # Return thresholded map
    return heatmap

def draw_labeled_bboxes(img, labels):
    # Iterate through all detected cars
    for car_number in range(1, labels[1]+1):
        # Find pixels with each car_number label value
        nonzero = (labels[0] == car_number).nonzero()
        # Identify x and y values of those pixels
        nonzeroy = np.array(nonzero[0])
        nonzerox = np.array(nonzero[1])
        # Define a bounding box based on min/max x and y
        bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
        # Draw the box on the image
        cv2.rectangle(img, bbox[0], bbox[1], (0,0,255), 6)
    # Return the image
    return img

def convert_color(img, conv='YCrCb'):
    if conv == 'HSV':
        return cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
    if conv == 'RGB2YCrCb':
        return cv2.cvtColor(img, cv2.COLOR_RGB2YCrCb)
    if conv == 'BGR2YCrCb':
        return cv2.cvtColor(img, cv2.COLOR_BGR2YCrCb)
    if conv == 'RGB2LUV':
        return cv2.cvtColor(img, cv2.COLOR_RGB2LUV)
    if conv == 'YCrCb':
        return cv2.cvtColor(img, cv2.COLOR_RGB2YCrCb)

def find_car_test(img, svc, X_scaler, orient=9, pix_per_cell=8, 
                  cell_per_block=2, spatial_size=(32, 32), hist_bins=32, 
                  conv='YCrCb'):
    if img.shape[0] != 64 or img.shape[1] != 64:
        print("resize")
        subimg = cv2.resize(img, (64,64))
    else:
        subimg = img
  
    n = extract_features([subimg], spatial_size=spatial_size,
                        hist_bins=hist_bins)
    print("n {}-{}:".format(len(n), n[0].shape))
    X = np.vstack((n)).astype(np.float64)
    print("Shape X: {}".format(X.shape))
    test_features = X_scaler.transform(X)
    # Plot an example of raw and scaled features
    fig = plt.figure(figsize=(12,4))
    plt.subplot(131)
    plt.imshow(subimg)
    plt.title('Original Image')
    plt.subplot(132)
    plt.plot(X[0])
    plt.title('Raw Features')
    plt.subplot(133)
    plt.plot(test_features[0])
    plt.title('Normalized Features')
    fig.tight_layout()
    plt.show()
    #test_features = X_scaler.transform(np.hstack((shape_feat, hist_feat)).reshape(1, -1))    
    test_prediction = svc.predict(test_features)
    return test_prediction

# Define a single function that can extract features using hog sub-sampling and make predictions
def find_cars(img, ystart, ystop, scale, svc, X_scaler, orient=9, pix_per_cell=8, 
              cell_per_block=2, spatial_size=(32, 32), hist_bins=32, conv='YCrCb', draw=False, draw_overlapping=False):
    if draw is True:
        draw_img = np.copy(img)
    else:
        draw_img = None
    
    img_tosearch = img[ystart:ystop,:,:]
    ctrans_tosearch = convert_color(img_tosearch, conv=conv)
    if scale != 1:
        imshape = ctrans_tosearch.shape
        ctrans_tosearch = cv2.resize(ctrans_tosearch, (np.int(imshape[1]/scale), np.int(imshape[0]/scale)))
        
    ch1 = ctrans_tosearch[:,:,0]
    ch2 = ctrans_tosearch[:,:,1]
    ch3 = ctrans_tosearch[:,:,2]

    # Define blocks and steps as above
    nxblocks = (ch1.shape[1] // pix_per_cell) - cell_per_block + 1
    nyblocks = (ch1.shape[0] // pix_per_cell) - cell_per_block + 1 
    nfeat_per_block = orient*cell_per_block**2
    
    # 64 was the orginal sampling rate, with 8 cells and 8 pix per cell
    window = 64
    nblocks_per_window = (window // pix_per_cell) - cell_per_block + 1
    cells_per_step = 2  # Instead of overlap, define how many cells to step
    nxsteps = (nxblocks - nblocks_per_window) // cells_per_step
    nysteps = (nyblocks - nblocks_per_window) // cells_per_step
    
    # Compute individual channel HOG features for the entire image
    hog1 = get_hog_features(ch1, orient, pix_per_cell, cell_per_block, feature_vec=False)
    hog2 = get_hog_features(ch2, orient, pix_per_cell, cell_per_block, feature_vec=False)
    hog3 = get_hog_features(ch3, orient, pix_per_cell, cell_per_block, feature_vec=False)
    
    box_list = []
    for xb in range(nxsteps):
        for yb in range(nysteps):
            ypos = yb*cells_per_step
            xpos = xb*cells_per_step
            # Extract HOG for this patch
            hog_feat1 = hog1[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 
            hog_feat2 = hog2[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 
            hog_feat3 = hog3[ypos:ypos+nblocks_per_window, xpos:xpos+nblocks_per_window].ravel() 
            hog_features = np.hstack((hog_feat1, hog_feat2, hog_feat3))

            xleft = xpos*pix_per_cell
            ytop = ypos*pix_per_cell

            # Extract the image patch
            subimg = cv2.resize(ctrans_tosearch[ytop:ytop+window, xleft:xleft+window], (64,64))
          
            # Get color features
            spatial_features = bin_spatial(subimg, size=spatial_size)
            _, _, _, _, hist_features = color_hist(subimg, nbins=hist_bins)

            # Scale features and make a prediction
            test_features = X_scaler.transform(np.hstack((spatial_features, hist_features, hog_features)).reshape(1, -1))   
            test_prediction = svc.predict(test_features)
            
            xbox_left = np.int(xleft*scale)
            ytop_draw = np.int(ytop*scale)
            win_draw = np.int(window*scale)
            if test_prediction == 1:    
                box_list.append(((xbox_left, ytop_draw+ystart), (xbox_left+win_draw,ytop_draw+win_draw+ystart)))
                if draw is True:
                    cv2.rectangle(draw_img,(xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart),(0,0,255),12) 

            if draw is True and draw_overlapping is True:
                cv2.rectangle(draw_img,(xbox_left, ytop_draw+ystart),(xbox_left+win_draw,ytop_draw+win_draw+ystart),(0,255,0),3) 
                
    return box_list, draw_img

def draw_labeled_bboxes(img, labels):
    # Iterate through all detected cars
    for car_number in range(1, labels[1]+1):
        # Find pixels with each car_number label value
        nonzero = (labels[0] == car_number).nonzero()
        # Identify x and y values of those pixels
        nonzeroy = np.array(nonzero[0])
        nonzerox = np.array(nonzero[1])
        # Define a bounding box based on min/max x and y
        bbox = ((np.min(nonzerox), np.min(nonzeroy)), (np.max(nonzerox), np.max(nonzeroy)))
        # Draw the box on the image
        cv2.rectangle(img, bbox[0], bbox[1], (0,0,255), 6)
    # Return the image
    return img
In [34]:
test_crop_images = [ load_img(x) for x in glob.glob("test_images/test1_*") ]
show_images(test_crop_images, ["test crop"]*len(test_crop_images))
In [35]:
import warnings
warnings.filterwarnings("ignore")
result = find_car_test(test_crop_images[0], clf, scaler_X)
print("First car result: {}".format(result))

result = find_car_test(test_crop_images[1], clf, scaler_X)
print("First car result: {}".format(result))

result = find_car_test(vehicle_images[0], clf, scaler_X)
print("Train car result: {}-{}".format(result, clf.predict(scaled_X[0])))
resize
n 1-(8460,):
Shape X: (1, 8460)
First car result: [ 1.]
resize
n 1-(8460,):
Shape X: (1, 8460)
First car result: [ 0.]
n 1-(8460,):
Shape X: (1, 8460)
Train car result: [ 1.]-[ 1.]
In [36]:
test_images = [ load_img(x) for x in glob.glob("test_images/*.jpg") ]
show_images(test_images, ["test"]*len(test_images))
In [37]:
import warnings
warnings.filterwarnings("ignore")
ystart = 400
ystop = 656
scale_list = [0.75, 1.0, 1.5, 2.0, 2.5]

image = test_images[0]
b_list, out_img = find_cars(image, ystart, ystop, scale_list[0], clf, scaler_X, draw=True, draw_overlapping=True)
fig = plt.figure()
plt.imshow(out_img)
plt.title("Sliding window")
fig.tight_layout()
fig.savefig("output_images/sliding_window_overlapped.png")
In [38]:
box_list = []
for s in scale_list:
    b_list, out_img = find_cars(image, ystart, ystop, s, clf, scaler_X, draw=True)
    box_list.extend(b_list)
In [39]:
show_images([find_cars(x, ystart, ystop, scale_list[0], clf, scaler_X, draw=True)[1] for x in test_images[0:5]],
            ["2+", "2+1-", "1+", "1-", "1-"], save="test_detection")
In [40]:
heat = np.zeros_like(image[:,:,0]).astype(np.float)

# Add heat to each box in box list
heat = add_heat(heat,box_list)
    
# Apply threshold to help remove false positives
heat = apply_threshold(heat,4)

# Visualize the heatmap when displaying    
heatmap = np.clip(heat, 0, 255)

labels = label(heat)

draw_img = draw_labeled_bboxes(np.copy(image), labels)

fig = plt.figure(figsize=(12,4))
plt.subplot(121)
plt.imshow(heatmap, cmap='hot')
plt.title('Heat Map')
plt.subplot(122)
plt.imshow(draw_img)
plt.title('Vechicle boundary')
fig.tight_layout()
fig.savefig("output_images/vehicle_heat_boundary.png")
In [41]:
# Import everything needed to edit/save/watch video clips
from moviepy.editor import VideoFileClip
from IPython.display import HTML

default_args = { 
    'ystart': 400,
    'ystop': 656,
    'scale_list':  [0.75, 1.0, 1.5, 2.0, 2.5],
    'thresh': 4,
    'color': 'YCrCb',
    'model': clf,
    'scaler_X': scaler_X }

def pipeline(img_rgb, args=default_args):
    box_list = []
    for scale in args['scale_list']:
        b_list, _ = find_cars(img_rgb, args['ystart'], args['ystop'], 
                              scale, args['model'], args['scaler_X'], 
                              conv=args['color'])
        box_list.extend(b_list)
    heat = np.zeros_like(img_rgb[:,:,0]).astype(np.float)

    # Add heat to each box in box list
    heat = add_heat(heat, box_list)
    
    # Apply threshold to help remove false positives
    heat = apply_threshold(heat, args['thresh'])

    return draw_labeled_bboxes(np.copy(img_rgb), label(heat))
In [42]:
def process_video(input_video, output_video, clip=None):
    white_output = 'output_videos/%s' % output_video
    ## To speed up the testing process you may want to try your pipeline on a shorter subclip of the video
    ## To do so add .subclip(start_second,end_second) to the end of the line below
    ## Where start_second and end_second are integer values representing the start and end of the subclip
    ## You may also uncomment the following line for a subclip of the first 5 seconds
    if clip is not None:
        clip1 = VideoFileClip('%s' % input_video).subclip(clip[0],clip[1])
    else:
        clip1 = VideoFileClip('%s' % input_video)

    def process_image(img):
        return pipeline(img)
    white_clip = clip1.fl_image(process_image) #NOTE: this function expects color images!!
    %time white_clip.write_videofile(white_output, audio=False)
    
    return white_output
In [43]:
white_output = process_video("test_video.mp4", "test_video.mp4")
[MoviePy] >>>> Building video output_videos/test_video.mp4
[MoviePy] Writing video output_videos/test_video.mp4
 97%|█████████▋| 38/39 [01:22<00:02,  2.18s/it]
[MoviePy] Done.
[MoviePy] >>>> Video ready: output_videos/test_video.mp4 

CPU times: user 1min 30s, sys: 470 ms, total: 1min 31s
Wall time: 1min 23s
In [44]:
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(white_output))
Out[44]:
In [45]:
white_output = process_video("project_video.mp4", "project_video.mp4")
[MoviePy] >>>> Building video output_videos/project_video.mp4
[MoviePy] Writing video output_videos/project_video.mp4
100%|█████████▉| 1260/1261 [45:29<00:02,  2.17s/it]
[MoviePy] Done.
[MoviePy] >>>> Video ready: output_videos/project_video.mp4 

CPU times: user 50min 8s, sys: 13.2 s, total: 50min 22s
Wall time: 45min 29s
In [46]:
HTML("""
<video width="960" height="540" controls>
  <source src="{0}">
</video>
""".format(white_output))
Out[46]: